// creating pronunciation dictionary from many tables
// importing respelling dictionaries from other formats
// creating respelling dictionaries them from pronunciation dictionaries

#include "rs_dict.h"
#include "rs_stream.h"
#include "rs_table.h"
#include "rs_convert.h"
#include "rs_respell.h"


// flags: puk,pus,                    : from UK or US dictionary
//        v<n>,                       : variation number
//        nc, np                      : common word (noun), proper word (noun)
//        fch, fsm                    : Flag CHanged, Flag SaMe: word was changed when respelling, word stayed the same
//        fbs                         : Flag BaSic: non-composed word, basic word
//        fsw                         : Flag Special Word


rs_string pron_ography::get_str() const
{
    rs_string to=po_to.get_str();
    if (po_to2.get_str().length()>0)
        to += L">" + po_to2.get_str();
    rs_string pr=po_pr.get_str();
    if (po_pr2.get_str().length()>0)
        pr += L">" + po_pr2.get_str();
    return to+L"/"+pr+L" "+
        rs_stringed(po_to_full)+L"/"+rs_stringed(po_pr_full); // traditional orthography/ pronunciation
}



static flag_text conv_ft[]={
    cf_start_word, L"sw",
    cf_end_word, L"ew",
    cf_before_stress, L"bs",
    cf_after_stress, L"as",
    0, 0
};


ro_conv_search::ro_conv_search (rs_string find_expr) :
    rocf_wflag (L""),
    rocf_nflags(0)
{
    // <conv_find> is [<wflag>:]<find>[;<nflags]
    rs_string nflags;
    unsigned colon_pos = find_expr.find(L":");
    if (colon_pos != string::npos) {
        rocf_wflag = find_expr.substr (0, colon_pos);
        find_expr = find_expr.substr (colon_pos+1);
    }
    unsigned semicolon_pos = find_expr.find(L";");
    if (semicolon_pos != string::npos) {
        string_to_flags (find_expr.substr (semicolon_pos+1), rocf_nflags, conv_ft, L"conversion", false);
        find_expr = find_expr.substr (0, semicolon_pos);
    }
    rs_string_v find_v;
    string_to_vec (find_expr, find_v, L"/");
    unsigned find_size = find_v.size(), i;
    for (i=0; i<find_size; i++) {
        rs_string find_regex = regex_add_start_end (find_v[i], true, true);
        rocf_search_v.push_back (rs_regex(find_regex, &cmudict_charset));
    }
    rocf_nsize = rocf_search_v.size();
}


rs_string ro_conv_search::search_str() const
{
    rs_string result;
    unsigned i, find_size = rocf_search_v.size();
    for (i=0; i<find_size; i++)
        result += (i>0? L"/": L"") + rocf_search_v[i].get_str();
    return result;
}


bool ro_conv_search::matches (const rs_string_v& nibble_v, unsigned start) const
{
    unsigned nibble_size = nibble_v.size();
    if (start + rocf_nsize > nibble_size)
        return false;
    if (rocf_nflags & cf_start_word)
        if (start > 0)
            return false;
    if (rocf_nflags & cf_end_word)
        if (start + rocf_nsize != nibble_size)
            return false;
    unsigned i;
    for (i=0; i<rocf_nsize; i++) {
        if (!rocf_search_v[i].matches (nibble_v[i+start]))
            return false;
    }
    return true;
}


ro_conv_rule::ro_conv_rule (const rs_string& search,
                  const rs_string& replace,
                  const rs_string& option)  :
    roc_replace (replace),
    roc_option (option),
    roc_active(false)
{

    rs_string_v search_v;
    string_to_vec (search, search_v, L"|");
    unsigned i, search_size=search_v.size();
    if (search_size != 1)
        search_size = search_size; // debugging only
    for (i=0; i<search_size; i++)
        roc_search_v.push_back (ro_conv_search (search_v[i]));
}


ro_option::ro_option (const rs_string& name)
{
    roo_nd_name = roo_name = name;
    replace_all (roo_nd_name, L"-", L"");
    to_cix = roo_word.add_text_col (L"WORD_TO", true, 10);
    ro_cix = roo_word.add_text_col (L"WORD_RO", false, 10);
    affix_cix = roo_affix.add_text_col (L"AFFIX", true, 10);
}


void ro_option::add_word (const rs_string& word_to, const rs_string& word_ro)
{
    roo_word.append_text (word_to, to_cix);
    roo_word.append_text (word_ro, ro_cix);
}


void ro_option::add_affix (const rs_string& affix)
{
    roo_affix.append_text (affix, affix_cix);
}


bool pobro::relate_rec (
    const rs_string word_to, // remaining TO word to be related
    const rs_string word_pr, // remaining PR word to be related
    unsigned priority,       // use only rules with po_priority <= priority
    rs_string& word_rel,      // if word_rel succeded, here is the result; if failed, shows the failure
    unsigned& max_len_failed,
    rs_string& failed) const
{
    // creates a relation (word_rel) between orthography word_to and pronunciation word_pr
    // returns true if relation was found using specified relation rules
    // recursive function
    if (word_to.length() == 0 && word_pr.length() == 0) {
        return true;
    }
    rs_string orig_mapping(word_rel);
    unsigned match_size= pob_rel.size();
    for (unsigned i=0; i<match_size; i++) {
        if (pob_rel[i].get_priority() <= priority &&
            pob_rel[i].matches (word_to, word_pr)) {
            word_rel = orig_mapping + L"/"
                + word_to.substr (0, pob_rel[i].get_po_to_len())
                + word_pr.substr (0, pob_rel[i].get_po_pr_len());
            if (relate_rec (
                word_to.substr (pob_rel[i].get_po_to_len()),
                word_pr.substr (pob_rel[i].get_po_pr_len()),
                priority,
                word_rel, max_len_failed, failed))
                return true;
        }
    }
    unsigned this_len_failed = orig_mapping.length();
    if (this_len_failed > max_len_failed) {
        max_len_failed = this_len_failed;
        failed = orig_mapping + L"/--" + word_to + L"--" + word_pr;
    }
    return false;
}



bool pobro::relate (
    const rs_string& word_to,
    const rs_string& word_pr,
    rs_string& word_rel) const
{
    // creates a word_rel between orthography word_to and pronunciation word_pr
    // returns true if full word_rel was found
    word_rel=L"";
    unsigned max_len_failed1 = 0, max_len_failed2 = 0;
    rs_string failed1=L"", failed2=L"";

    bool result1 = relate_rec (word_to, word_pr, 1, word_rel, max_len_failed1, failed1);
    bool result2 (false);
    if (!result1) {
        word_rel=L"";
        result2 = relate_rec (word_to, word_pr, 2, word_rel, max_len_failed2, failed2);
    }
    if (word_rel.substr(0,1) == L"/")
        word_rel = word_rel.substr (1);
    return result1 || result2;
}

void pobro::sort_pob_rel()
{
    // stably sorts vector<pron_ography> pob_rel into order in which they're to be processed
    // class pron_ography is partially ordered by function longer()
    // stable sort: function sort_pob_rel keeps relative order between 
    // elements a=pob_rel[i] and b=pob_rel[j]
    // for which neither a.longer(b) nor b.longer(a) is true
    // quadratic algorithm O(pob_rel.size()), sorry, I don't know anything better
    // for stable sorting using partial ordering
    vector<pron_ography> sorted;
    unsigned size=pob_rel.size(), i, j, longest;
    vector<bool> free_elem (size, true);
    log_pob_rel (L"unsorted - as listed");
    for (i=0;  i<size;  i++) {
        // sorting by choosing the longest element from pob_rel[i..end]
        // for which free_elem[j] is true
        j=0;
        while (!free_elem[j])
            j++;
        longest = j;
        for (j=longest+1;  j<size;  j++) {
            if (pob_rel[j].longer (pob_rel[longest]) && free_elem[j])
                longest = j;
        }
        // copy the longest element to "sorted" and mark it in "free_elem"
        sorted.push_back (pob_rel[longest]);
        free_elem[longest] = false;
    }
    pob_rel = sorted;
    log_pob_rel (L"sorted - as processed");
}

void pobro::log_pob_rel (const rs_string& title) const
{
    unsigned size=pob_rel.size(), i;
    rs_log_file (L"List of connections (regexes) for pronunciation dictionary: " + title);
    for (i=0;  i<size;  i++) {
        const pron_ography& po=pob_rel[i];
        rs_log_file (L"Regex TO-PR: " + rs_stringed(i) + L":" + po.get_str());
    }
}


bool pobro::parse_pron_conv (const rs_string& pron_conv_fname)
{ // reading file describing pronunciation dictionary analysis and conversion
  // one such file is "u1_pron_conv.rst"
    rs_log_console (L"Reading and processing file " + pron_conv_fname);
    rs_table pron_conv_rst (pron_conv_fname);
    if (!pron_conv_rst.is_valid()) {
        return false;
    }
    if (pron_conv_rst.get_col_count() < 3 ||
        pron_conv_rst.get_col_type (any_cix(0)) != rt_text2  ||
        pron_conv_rst.get_col_type (any_cix(1)) != rt_text2  ||
        pron_conv_rst.get_col_type (any_cix(2)) != rt_text2) {
        rs_report_error (L"Incorrect column types in pronunciation convertion table /TTT/: ", pron_conv_fname);
        return false;
    }
    
    rs_rix cm_rrows = pron_conv_rst.get_row_count();
    enum cr_section { crs_pairs, crs_skip, crs_option};  // three main sections of _pron_conv.rst file
    enum cr_option { cro_convert, cro_word, cro_affix};  // subsections of "option" section
    cr_section crs=crs_pairs;
    cr_option cro=cro_convert;
    rs_string option;
    unsigned priority = 1;
    for (rs_rix rix=rs_rix::begin(); rix<cm_rrows; rix++) {
        rs_string t0 (pron_conv_rst.get_text (text_cix(0), rix));
        rs_string t1 (pron_conv_rst.get_text (text_cix(1), rix));
        rs_string t2 (pron_conv_rst.get_text (text_cix(2), rix));
        if (t0.length() == 0)
            continue;
        if (t0 == L"priority:") {
            priority = string_to_unsigned (t1);
            continue;
        }
        if (t0 == L"skip:") {
            crs = crs_skip;
            continue;
        }
        if (t0 == L"option:") {
            // some example options names: e, u-u, u-h
            // when creating file names and abbreviations for ROs, dash is skipped
            // if dash is specified, text before dash is option group; only one
            // option from a group can be applied when transforming TO words
            // option names can be from ASCII letters, digits, dash, and underline; names are case insensitive
            crs = crs_option;
            cro = cro_convert;
            option = t1;
            to_lower(option);
            if (!option_exists (option)) {
                pob_option_v.push_back (ro_option(option));
                rs_string opt_group = nth_substring (option, 0, L"-");
                pob_opt_gr_v.push_back (opt_group);
            }
            continue;
        }
        if (t0 == L"convert:") {
            if (crs != crs_option) {
                rs_report_error (L"Unexpected sub-section start, it can only appear in 'option' section: "
                    + t0);
                continue;
            }
            cro = cro_convert;
            continue;
        }
        if (t0 == L"special_words:") {
            if (crs != crs_option) {
                rs_report_error (L"Unexpected sub-section start, it can only appear in 'option' section: "
                    + t0);
                continue;
            }
            cro = cro_word;
            ro_option& opt = get_option (option);
            rs_string_v word_v;
            string_to_vec (t1, word_v, L",");
            unsigned i, word_size = word_v.size();
            for (i=0; i<word_size; i++) {
                rs_string to_form, ro_form;
                to_form = word_v[i];
                ro_form = nth_substring (to_form, 1, L">");
                to_form = nth_substring (to_form, 0, L">");
                opt.add_word (to_form, ro_form);
            }
            continue;
        }
        if (t0 == L"affix:") {
            if (crs != crs_option) {
                rs_report_error (L"Unexpected sub-section start, it can only appear in 'option' section: "
                    + t0);
                continue;
            }
            cro = cro_affix;
            continue;
        }
        if (t0 == L"stress:") {
            pob_stress_option = option;
            if (t1.length() % 2 == 0)
                pob_stress_pairs += t1;
            else {
                rs_report_error (L"unrecognized stress specification - size not even",
                    t1);
            }
            continue;
        }
        if (crs == crs_pairs) {
            pob_rel.push_back(pron_ography(t0, t1, priority));
            continue;
        }
        if (crs == crs_skip) {
            to_lower (t0);
            pob_skipped.insert (t0);
            continue;
        }
        if (crs == crs_option  && cro == cro_convert) {
            pob_conv.push_back (ro_conv_rule (t0, t1, option));
            if (!t2.empty()  && t2[0]!=L'#') 
                rs_report_error (L"only two fields expected for conversion rule, but more present: ",
                    t0 + L" " + t1 + L" " + t2);
            continue;
        }
        if (crs == crs_option  && cro == cro_word) {
            ro_option& opt = get_option (option);
            opt.add_word (t0, t1.empty()? t0:t1);
            continue;
        }
        if (crs == crs_option  && cro == cro_affix) {
            ro_option& opt = get_option (option);
            opt.add_affix (t0);
            continue;
        }
        rs_report_error (L"internal error: crs value");
    }
    sort_pob_rel();
    return true;
}


void remove_dashes (rs_string_v& opt_v)
{   // from the vector of options, removes all dashes "-"
    unsigned opt_size=opt_v.size(), i;
    for (i=0; i<opt_size; i++) {
        replace_all (opt_v[i], L"-", L"");
    }
}



void pobro::enable_options (rs_string_v option_v)
{
    // sets flag "enabled" for all conversions in listed options
    // concatenates all roo_affix and roo_word table
    rs_log_file (L"enabling conversion options: ");
    remove_dashes (option_v);
    pob_mark_stress = false;
    unsigned opt_size=option_v.size(), i;
    pob_word.clear();
    pob_affix.clear();
    for (i=0; i<opt_size; i++) {
        rs_log_file (L"option_v[" + rs_stringed(i) + L"]=" + option_v[i]);
        ro_option& opt = get_option (option_v[i], true);
        if (pob_word.get_col_count() == 0)
            pob_word.get_columns_from (opt.get_word_rst());
        pob_word.add_all_rows (opt.get_word_rst(), pob_word.get_col_count());
        if (pob_affix.get_col_count() == 0)
            pob_affix.get_columns_from (opt.get_affix_rst());
        pob_affix.add_all_rows (opt.get_affix_rst(), pob_affix.get_col_count());
    }
    pob_word.create_hashes();
    pob_affix.create_hashes();
    unsigned conv_size = pob_conv.size();
    for (i=0;  i<conv_size; i++) {
        rs_string opt = pob_conv[i].get_option();
        replace_all (opt, L"-", L"");
        //rs_log_file (L"checking " + opt);
        bool present = find (option_v.begin(), option_v.end(), opt) != option_v.end();
        pob_conv[i].set_active (present);
        if (present) {
            rs_log_file (L"enabled: "
                + pob_conv[i].search_str() + L" ==> "
                + pob_conv[i].get_replace()
                );
        }
    }
    if (find (option_v.begin(), option_v.end(), pob_stress_option) != option_v.end()) {
        pob_mark_stress = true;
        rs_log_file (L"enabled: stress marking " + pob_stress_pairs);
    }
}


bool pobro::option_exists (const rs_string& option)
{
    unsigned option_size = pob_option_v.size();
    for (unsigned i=0; i< option_size; i++) {
        if (pob_option_v[i].get_name() == option)
            return true;
    }
    return false;
}


ro_option& pobro::get_option (const rs_string& option, bool no_dashes)
{
    unsigned option_size = pob_option_v.size();
    for (unsigned i=0; i< option_size; i++) {
        if (no_dashes) {
            // 'option' is without dashes
            if (pob_option_v[i].get_nd_name() == option)
                return pob_option_v[i];
        } else {
            if (pob_option_v[i].get_name() == option)
                return pob_option_v[i];
        }
    }
    rs_report_error (L"Unknown option: " + option);
    return pob_option_v[0];
}


rs_string_v pobro::get_all_options() const
{
    rs_string_v all_options;
    unsigned option_size = pob_option_v.size();
    for (unsigned i=0; i< option_size; i++) {
        all_options.push_back (pob_option_v[i].get_name());
    }
    return all_options;
}


rs_string get_replacement (
    const rs_string_v& nibble_v,
    unsigned i, // initial nibble
    unsigned i_incr, // number of nibbles
    const rs_string& replace_pattern)
{
    // get string that replaces some nibbles
    // - starting with nibble_v[i], total of i_incr nibbles
    // according the the replace pattern
    rs_string_v replace_v;
    string_to_vec (replace_pattern, replace_v, L"/");
    unsigned replace_size = replace_v.size();
    rs_string result;
    for (unsigned ii=0; ii<i_incr; ii++) {
        if (ii>=replace_size)
            break;
        rs_string nibble = nibble_v[i+ii], rnibble;
        rs_string replace = replace_v[ii];
        if (replace.substr(0,2)==L"\\0")
            rnibble = nth_substring (nibble, 0, L"_");
        else
        if (replace.substr(0,2)==L"\\1")
            rnibble = nth_substring (nibble, 1, L"_");
        else
        if (replace.substr(0,2)==L"\\2")
            rnibble = nth_substring (nibble, 2, L"_");
        else
            rnibble = replace;
        result += rnibble;
    }
    return result;
}


bool matches_additional (
    const rs_table& pron_rst,
    const vector<rs_rix>& word_to_rix_v,
    const vector<rs_regex>& search_v,
    const rs_string& search_wflag,
    unsigned nix)
    // searches all rows 'word_to_rix_v' in table 'pron_rst' to find any word that
    // matches 'search_v' starting from nibble 'nix' on and has word flag 'wflag'
    // if found any word, returns true
    // otherwise returns false
{
    unsigned i, rix_size = word_to_rix_v.size();
    unsigned search_size = search_v.size();
    rs_rix rix;
    text_cix cix_or_pr = pron_rst.get_text_cix (L"OR_PR");
    text_cix cix_flags = pron_rst.get_text_cix (L"FLAGS");
    for (i=0; i< rix_size; i++) {
        rix = word_to_rix_v[i];
        const rs_string& word_rel=pron_rst.get_text (cix_or_pr, rix);
        rs_string_v nibble_v; // vector of nibbles
        string_to_vec (word_rel, nibble_v, L"/");
        unsigned j,nibble_size = nibble_v.size();
        rs_string_v wflag_v;
        if (!search_wflag.empty()) {
            string_to_vec (pron_rst.get_text (cix_flags, rix), wflag_v, L",");
            if (find(wflag_v.begin(), wflag_v.end(), search_wflag) == wflag_v.end()) {
                continue;
            }
        }
        bool found = true;
        for (j=0; j< search_size; j++) {
            if (nix+j >= nibble_size  ||  !search_v[j].matches (nibble_v[nix+j])) {
                found = false;
                break;
            }
        }
        if (found)
            return true;
    }
    return false;
}



void pobro::convert_to_ro_base (
    const rs_string& word_to,
    rs_string word_rel,
    rs_string& word_ro,
    unsigned& nib_chan,
    unsigned& nib_all,
    unsigned used_conv_ix,
    bool& used,
    rs_string& used_nibble,
    const rs_string& stress,
    const rs_table& pron_rst) const
{
    // used for producing RO dictionary or to get statistics about a conversion rule

    // word_rel is something like L"/g_g/a_ey/m_m/e" that links
    // "game" with its pronunciation "_g_ey_m"
    // if pob_mark_stress, replaces the first replacable character (typically, aeiouAEIOU)
    // in stressed syllable/nibble with wanted stress as stored in pob_stress_pairs
    // 'used' is set to true if conversion used index 'used_conv_ix'
    // 'stress' - if known - has format: fixed character "s", then index of stressed nibble, then slash, then the rest
    nib_chan = 0;
    rs_string_v nibble_v; // vector of nibbles
    string_to_vec (word_rel, nibble_v, L"/");
    unsigned nibble_size = nibble_v.size();
    nib_all = nibble_size;
    used = false;
    used_nibble = L"";
    unsigned ix_stress(nibble_size);  // index of stressed nibble
    if (pob_mark_stress) {
        rs_string ix_str = nth_substring (stress, 0, L"/");
        if (ix_str.substr(0,1) == L"s") {
            ix_str = ix_str.substr(1);
            if (!ix_str.empty())
                ix_stress = string_to_unsigned (ix_str);
        }
    }
    word_ro = L"";
    for (unsigned i=0; i<nibble_size; ) {
        const rs_string& nibble = nibble_v[i];
        if (nibble.length() == 0)
            continue;
        // if nibble conversion is found in pob_conv and enabled, it is converted
        unsigned j, k, cnv_size=pob_conv.size();
        rs_string nibble_ro; // respelled-ortography form
        unsigned i_incr=0; // number of nibbles to increment 'i'
        bool nibble_converted(false);
        for (j=0; j<cnv_size; j++) {
            const ro_conv_rule& pob_conv_j = pob_conv[j];
            if (pob_conv_j.get_active() && pob_conv_j.matches (nibble_v, i)) {
                unsigned scount = pob_conv_j.get_search_count();
                if (scount > 1) {
                    // get list of all rows with same word_to from pron_rst
                    vector<rs_rix> word_to_rix_v = pron_rst.get_text_row_all (word_to, text_cix(0));
                    bool passed_add = true;
                    for (k=1; k<scount; k++) {
                        // satisfy k-th search condition
                        if (!matches_additional (pron_rst, word_to_rix_v, 
                                pob_conv_j.get_search_v(k), pob_conv_j.get_wflag(k), i)) {
                            passed_add = false;
                            break;
                        }
                    }
                    if (!passed_add)
                        continue; // additional conditions were not met
                }
                const rs_string& replace_pattern = pob_conv_j.get_replace();
                i_incr = pob_conv_j.get_search_nsize();
                nibble_ro = get_replacement (nibble_v, i, i_incr, replace_pattern);
                nib_chan++;
                nibble_converted=true;
                if (j == used_conv_ix) {
                    used = true;
                    used_nibble = nibble_v[i];
                    unsigned k, k_max = i+i_incr;
                    if (k_max > nibble_size)
                        k_max = nibble_size;
                    for (k=i+1; k<k_max; k++)
                        used_nibble += L"/" + nibble_v[k];
                }
                break;
            }
        }
        if (!nibble_converted) {
            nibble_ro = nth_substring (nibble, 0, L"_");
            i_incr = 1;
        }
        if (i == ix_stress) {
            // this is stressed nibble that will be marked
            // search for vocal to be replaced
            unsigned stress_len = pob_stress_pairs.length();
            for (unsigned j=0; j<stress_len; j+=2) {
                unsigned stress_pos = nibble_ro.find (pob_stress_pairs[j]);
                if (stress_pos != string::npos) {
                    // found stress data
                    nibble_ro[stress_pos] = pob_stress_pairs[j+1];
                    break;
                }
            }
        }
        word_ro += nibble_ro;
        i += i_incr;
    }
}


bool import::convert_load_freq (const rs_string& freq_fname)
{
    // loading frequency table and adding column FREQL
    rs_log_console (L"Reading and processing word frequency table " + freq_fname);
    if (freq_fname != L""  &&  freq_fname!=L".") {
        imp_freq_rst.read (freq_fname);
        if (!imp_freq_rst.is_valid()) {
            rs_report_error (L"Problems opening word frequency table ", freq_fname);
            return false;
        }
    }
    imp_freq_all_cix = imp_freq_rst.get_numb_cix (L"FR_ALL");
    imp_freq_lc_cix = imp_freq_rst.get_numb_cix (L"FR_LC");
    if (!imp_freq_all_cix.is_valid()  || !imp_freq_lc_cix.is_valid()) {
        rs_report_error (L"Problems processing frequency table: "
            L"missing numerical column(s) FR_ALL and/or FR_LC");
        return false;
    }

    // calculate column "FREQL" for table imp_freq_rst
    // it is calculated from column given its name
    // frequency class is logarithm with base 2 of
    // ratio between the most occuring word ("the") and specified word
    imp_freq_frc_cix = imp_freq_rst.add_numb_col (L"FREQL", 2);
    unsigned max_freq=0;
    rs_rix end_rix = imp_freq_rst.get_row_count();
    // let's get the maximal frequency into max_freq
    for (rs_rix rix=rs_rix::begin();  rix<end_rix;  rix++) {
        unsigned this_freq = imp_freq_rst.get_numb (imp_freq_all_cix, rix);
        if (max_freq < this_freq)
            max_freq = this_freq;
    }
    // calculate data for column FREQL for every row
    for (rs_rix rix=rs_rix::begin();  rix<end_rix;  rix++) {
        unsigned this_freq = imp_freq_rst.get_numb (imp_freq_all_cix, rix);
        unsigned freql = 0;
        for (unsigned fr=max_freq; fr>this_freq; fr/=2)
            freql++;
        imp_freq_rst.set_numb (freql, imp_freq_frc_cix, rix);
    }
    return true;
}


static void read_25 (rs_istream &ifs, rs_string &wcs_wd)
{ // reads 25 characters from 'ifs' into 'wf'
    char dlf[26];
    ifs.read (dlf, 25);
    for (int i=24; i>=0; i--) {
        if (dlf[i]!= ' ' && dlf[i]!='\0')
            break;
        dlf[i]='\0';
    }
    string str_wd(dlf);
    ascii_to_wstring (str_wd, wcs_wd);
}


bool import::convert_d50 (
                  const rs_string& in_fname, 
                  const rs_string& freq_fname)
{
    if (!convert_load_freq (freq_fname))
        return false;
    rs_log_console (L"Reading and processing dictionary " + in_fname);
    imp_ifs.open(in_fname);
    if (!imp_ifs.good()) {
        rs_report_error (L"Problems opening input file ", in_fname);
        return false;
    }

    rs_string word_to, word_ro;
    while (!imp_ifs.eof()) {
        read_25 (imp_ifs, word_to);
        read_25 (imp_ifs, word_ro);
        if (word_to.compare (L"zzz") == 0)
            break;

        bool several;
        rs_rix freq_rix = imp_freq_rst.get_text_row_any (word_to, text_cix(0), several);
        unsigned freq_all=0, word_frc=99;
        if (freq_rix.is_valid()) {
            freq_all = imp_freq_rst.get_numb (imp_freq_all_cix, freq_rix);
            word_frc = imp_freq_rst.get_numb (imp_freq_frc_cix, freq_rix);
        }
        if (word_to.length() != 0) {
            imp_out_rst.append_text (word_to, imp_cix_or_to);
            imp_out_rst.append_text (word_ro, imp_cix_or_ro);
            imp_out_rst.append_numb (freq_all, imp_cix_freq);
            //imp_out_rst.append_numb (word_frc, imp_cix_frc);
            imp_out_rst.append_text (rs_string(L""), imp_cix_flags);
        }
    }
    imp_out_rst.write (imp_out_fname);
    rs_log_console (L"Writing output dictionary " + imp_out_fname);
    return true;
}



bool import::pron_dict_combine (
        const rs_string& us_fname, // pronouncing table for US english
        const rs_string& uk_fname, // pronouncing table for UK english
        const rs_string& freq_fname,
        const rs_string& pron_conv_fname,
        const rs_string& case_fname,
        const rs_string& affix_cg_fname,
        const rs_string& start_word,
        unsigned row_limit,
        bool incl_us,
        bool incl_uk)
{ // importing pronouncing dictionary based on several files:
    // 1) pronouncing tables pd_us.rst and pd_uk.rst based on "cmudict.06d" and "beep-1.0"
    // 2) frequency table (freq_fname)
    // 3) pronouncing conversion rules file (pron_conv_fname)
    if (!convert_load_freq (freq_fname))
        return false;
    imp_out_rst.set_col_min_width (40, imp_cix_or_ro);
    imp_start_word = start_word;
    imp_row_limit = row_limit;
    pobro pob;
    if (!pob.parse_pron_conv (pron_conv_fname)) { // e.g. L"u1_pron_conv.rst"
        rs_report_error (L"problems reading pronouncing conversion table ", pron_conv_fname);
        return false;
    }
    rs_log_console (L"Reading and processing word case table " + case_fname);
    rs_table case_rst;
    if (case_fname != L""  &&  case_fname!=L".") {
        case_rst.read (case_fname);
        if (!case_rst.is_valid()) {
            rs_report_error (L"Problems opening word frequency table ", case_fname);
            return false;
        }
    }
    rs_affix aff (affix_cg_fname, L"TO");
    if (incl_us)
        convert_one_pron (us_fname, L"pus", pob, case_rst, aff);
    if (incl_uk)
        convert_one_pron (uk_fname, L"puk", pob, case_rst, aff);

    rs_log_console (L"Writing pronouncing dictionary " + imp_out_fname);
    imp_out_rst.write (imp_out_fname, L"", L"", L"+or_to_us");
    return true;
}


void import::extract_stress (rs_string& word_pr, rs_string& stress_data)
{
    // divides word_pr into nibbles and for each nibble gets its stress_data
    // then removes stress data from word_pr
    rs_string_v word_v;
    string_to_vec (word_pr, word_v, L"_");
    unsigned vec_size = word_v.size(), i;
    stress_data = L"";
    bool specified(false);
    unsigned max_ix=0;  // index of nibble with highest stress
    wchar_t max_value =L' ';  // highest stress value
    for (i=0; i<vec_size; i++) {
        rs_string& nibble=word_v[i];
        unsigned nibble_len=nibble.length();
        if (nibble_len == 0)
            continue;
        wchar_t last=nibble[nibble_len-1];
        if (wcschr (L"012", last)) {
            stress_data += last;
            nibble.erase (nibble_len - 1);
            specified = true;
            if (last > max_value) {
                max_value = last;
                max_ix = i;
            }
        } else
            stress_data += L'c';
    }
    vec_to_string (word_v, word_pr, L"_");
    if (!specified)
        stress_data = L"na";
    else
        stress_data = L"s" + rs_stringed (max_ix) + L"/" + stress_data;
}


bool import::convert_one_pron (
        const rs_string& in_fname, // pronouncing table for US or UK english
        const rs_string& us_uk_flag,
        const pobro& pob,
        const rs_table& case_rst,
        const rs_affix& aff
        )
{ // converting pronouncing table 'in_fname'
    rs_log_console (L"Processing dictionary table, it can take few minutes: "+in_fname);
    rs_table in_rst (in_fname, false);
    rs_table out_rst;
    out_rst.get_columns_from (imp_out_rst);
    rs_string line, variation, word_to, word_pr, word_dec, word_decx;
    rs_rix word_ct = in_rst.get_row_count();
    rs_string prev_word;  // beep-1.0 doesn't mark variations, so they need to be detected
    unsigned var_count=1;
    bool start_word_passed = imp_start_word == L"";  // flag: processing has started as 'start_word' has been passed
    unsigned row_count = 0;  // so many words processed
    unsigned this_word_len, max_word_len=0;   // lengths of TO words
    bool store_intermediate_stages = true;
    /////////////////////////////////////////////////////////////////////
    // pass 1: split variation from the rest of word, split stress from pronunciation
    // get word frequency from frequency table
    // relating pronunciation with spelling
    for (rs_rix rix=rs_rix::begin();  rix<word_ct;  rix++) {
        word_to = in_rst.get_text (text_cix(0), rix);
        if (word_to.length()==0  ||  !is_alpha (word_to[0]))
            continue;
        if (!start_word_passed) {
            start_word_passed = word_to >= imp_start_word;
        }
        if (!start_word_passed) {
            continue;
        }
        if (imp_row_limit > 0 && row_count>=imp_row_limit)
            break;
        row_count++;
        to_lower (word_to);
        size_t opar_pos = word_to.find (L"(");
        size_t cpar_pos = word_to.find (L")");
        variation = L"";
        if (opar_pos != string::npos  &&  cpar_pos != string::npos
            && cpar_pos==opar_pos+2) {
            variation = word_to[opar_pos+1];
            word_to = word_to.substr (0, opar_pos);
        } else if (opar_pos != string::npos  ||  cpar_pos != string::npos) {
            rs_report_error (L"unrecognized TO word (.): ", word_to);
            continue;
        }
        if (word_to == prev_word) {
            var_count ++;
            variation = rs_stringed (var_count);
        } else {
            prev_word = word_to;
            var_count = 1;
        }
        if (!is_alpha (word_to)) {
            continue; // skipping such words
        }
        if (pob.skipped(word_to)) {
            rs_log_file (L"skipped word (by pron_conv rule): ", word_to);
            continue;
        }
        word_pr = in_rst.get_text (text_cix(1), rix);
        to_lower (word_pr);
        rs_string stress_data;
        extract_stress (word_pr, stress_data);
        if (variation.length() != 0)
            variation = L"v" + variation;
        rs_string flags(us_uk_flag);
        flags = concat (flags, variation, L',');
        bool several;
        rs_rix freq_rix = imp_freq_rst.get_text_row_any (word_to, text_cix(0), several);
        unsigned freq_all=0, word_frc=99, freq_lc=0;
        if (freq_rix.is_valid()) {
            freq_all = imp_freq_rst.get_numb (imp_freq_all_cix, freq_rix);
            //freq_lc =  imp_freq_rst.get_numb (imp_freq_lc_cix,  freq_rix);
            //word_frc = imp_freq_rst.get_numb (imp_freq_frc_cix, freq_rix);
        }
        this_word_len = word_to.length();
        if (max_word_len < this_word_len)
            max_word_len = this_word_len;
        word_pr = L'_' + word_pr;
        rs_string word_rel;
        bool related = pob.relate (word_to, word_pr, word_rel);
        /*
        if (!related  && variation == L"") {
            // place for breakpoint, debugging only
            related = related;
        }
        */
        rs_log_file (L"related: ", word_to 
            +L", " + word_pr 
            +L", " + word_rel 
            +L", flags=" + flags);

        out_rst.append_text (word_to, imp_cix_or_to);
        out_rst.append_text (word_rel, imp_cix_or_ro);
        out_rst.append_numb (freq_all, imp_cix_freq);
        out_rst.append_text (flags, imp_cix_flags);
        out_rst.append_text (stress_data, imp_cix_stress);
        // initialize next columns
        out_rst.append_text (L"", imp_cix_comp);
        out_rst.append_numb (0, imp_cix_freqx);
        out_rst.append_numb (0, imp_cix_wf_size);
        out_rst.append_text (L"", imp_cix_wf_base);
        out_rst.append_text (L"", imp_cix_affixes);
    }
    if (store_intermediate_stages)
        out_rst.write (L"pass1.rst");
    /////////////////////////////////////////////////////////////////////////////////
    // pass 2: the first variation (among multiple ones) doesn't have value "1", so let's update it
    // also use case_rst for determining common/proper flag
    rs_rix rix, rows=out_rst.get_row_count();
    rs_string var_one(L"v1");
    bool several;
    for (rix=rs_rix::begin(); rix<rows; rix++) {
        rs_string these_flags = out_rst.get_text (imp_cix_flags,rix);
        bool var_flag (false); // true if flags contain any "variation" flag i.e. "v1", "v2", .. "v
        rs_string_v flag_v;
        string_to_vec (these_flags, flag_v, L",");
        unsigned i, flag_size = flag_v.size();
        for (i=0; i<flag_size; i++) {
            if (flag_v[i].substr(0,1) == L"v") {
                var_flag = true;
                break;
            }
        }
        if (!var_flag) {
            these_flags = concat (these_flags, var_one, L',');
        }
        word_to = out_rst.get_text (imp_cix_or_to, rix);
        to_upper (word_to); // case_rst has uppercase values in column 0
        vector<rs_rix> case_v = case_rst.get_text_row_all (word_to, text_cix (0));
        bool is_lcase(false), is_ucase(false);
        unsigned case_size=case_v.size();
        for (i=0; i<case_size; i++) {
            const rs_string& word_cased = case_rst.get_text (text_cix(1), case_v[i]);
            if (word_cased.size() == 0)
                continue;
            if (is_lower (word_cased[0]))
                is_lcase=true;
            if (is_upper (word_cased[0]))
                is_ucase=true;
        }
        if (is_ucase)
            these_flags = concat (these_flags, L"np", L','); // proper word (noun)
        if (is_lcase)
            these_flags = concat (these_flags, L"nc", L','); // common word (noun)
        out_rst.set_text (these_flags, imp_cix_flags, rix);
    }
    if (store_intermediate_stages)
        out_rst.write (L"pass2.rst");

    ////////////////////////////////////////////////////////////////////
    // pass 3: decomposing TO words: removing affixes, spliting composed words
    unsigned word_len_ix;
    for (word_len_ix = 1;  word_len_ix < max_word_len;  word_len_ix++)
    {
        for (rix=rs_rix::begin(); rix<rows; rix++) {
            word_to = out_rst.get_text (imp_cix_or_to, rix);
            this_word_len = word_to.length();
            if (this_word_len != word_len_ix)
                continue; // processing only words of length word_len_ix
            if (word_to == L"abasers") {
                word_to = word_to;  // debugging only
            }
            if (word_to.find (imp_start_word) == 0) {
                word_to = word_to;  // debugging only
            }

            word_dec = word_to;
            vector<size_t> af_vec;
            wstring stem_src;
            if (aff.split_affixes (word_to, stem_src, af_vec, out_rst.get_column_ref(any_cix(0)), 2, 1) 
                && af_vec.size() > 0) {
                    word_dec = aff.show_affixes (word_to, stem_src, af_vec);
            }

            // todo : get split base word and affix as this is needed below
            bool fl_base = word_to == word_dec;
            rs_string flags = out_rst.get_text (imp_cix_flags,rix);
            rs_string wf_base, affixes;
            if (fl_base) {
                // no known affixes for this word, base word
                // todo: check whether word_to is composed of two or more words
                flags = concat (flags, L"fbs", L',');
                wf_base = word_to;
            } else {
                rs_string base1 = nth_substring (word_dec, 0, L","); // immediate base of word_to
                rs_string this_affix = nth_substring (word_dec, 1, L",");
                rs_rix rix_base1 = out_rst.get_text_row_any (base1, text_cix(0));
                if (rix_base1.is_valid()) {
                    wf_base = out_rst.get_text (imp_cix_wf_base, rix_base1);
                    affixes = concat (out_rst.get_text (imp_cix_affixes, rix_base1), 
                        this_affix, ',');
                } else {
                    wf_base = L"n/a"; // e.g. when not processing the whole table
                    affixes = L"n/a";
                }
            }
            out_rst.set_text (flags, imp_cix_flags, rix);
            out_rst.set_text (word_dec, imp_cix_comp, rix);
            out_rst.set_text (wf_base, imp_cix_wf_base, rix);
            out_rst.set_text (affixes, imp_cix_affixes, rix);
        }
    }
    // sum the frequencies into FREQX column
    for (rix=rs_rix::begin(); rix<rows; rix++) {
        rs_string these_flags = out_rst.get_text (imp_cix_flags,rix);
        word_to = out_rst.get_text (imp_cix_or_to, rix);
        out_rst.get_text_row_any (word_to, imp_cix_or_to, several);
        bool is_var1 = true;  // after calculation, true if this is first or only variation of word 
        if (several) {
            rs_string_v flag_v;
            string_to_vec (these_flags, flag_v, L",");
            is_var1 = find (flag_v.begin(), flag_v.end(), var_one) != flag_v.end();
        }
        unsigned i, base_size;
        if (is_var1) {
            const rs_string& base_word = out_rst.get_text (imp_cix_wf_base, rix);
            vector<rs_rix> rix_base_v = out_rst.get_text_row_all (
                base_word, imp_cix_or_to);
            unsigned freq_this = out_rst.get_numb (imp_cix_freq, rix);
            base_size = rix_base_v.size();
            for (i=0; i<base_size; i++) {
                rs_rix rix_base = rix_base_v[i];
                out_rst.incr_numb (freq_this, imp_cix_freqx, rix_base);
                out_rst.inc_numb (imp_cix_wf_size, rix_base);
            }
        }
        out_rst.set_text (these_flags, imp_cix_flags, rix);
    }
    if (store_intermediate_stages)
        out_rst.write (L"pass3.rst");
    rs_log_file (L"Transferring pronouncing dictionary to common table");
    imp_out_rst.add_all_rows (out_rst, out_rst.get_col_count());
    return true;
}


bool dcreate::init_tables (const rs_string& bpd_name,
                         const rs_string& pron_conv_fname,
                         const rs_string& pron_dict_fname)
{
    cd_bpd_name = bpd_name;
    cd_pron_conv_fname = pron_conv_fname;
    cd_pron_dict_fname = pron_dict_fname;
    if (cd_pron_conv_fname.length() == 0)
        cd_pron_conv_fname = bpd_name + L"_pron_conv.rst";
    if (cd_pron_dict_fname.length() == 0)
        cd_pron_dict_fname = bpd_name + L"_pron_dict.rst";
    rs_log_console (L"Reading files " + cd_pron_conv_fname + L" and " + cd_pron_dict_fname);
    if (!cd_pob.parse_pron_conv (cd_pron_conv_fname)) { // e.g. L"u1_pron_conv.rst"
        rs_report_error (L"problems reading pronunciation conversion table ", cd_pron_conv_fname);
        return false;
    }
    cd_pron_rst.read (cd_pron_dict_fname);
    if (!cd_pron_rst.is_valid()) {
        rs_report_error (L"problems reading table ", pron_dict_fname);
        return false;
    }
    return true;
}


bool dcreate::create_top (const rs_string& bpd_name,
                            rs_string options,
                            const rs_string& cdir,
                            const rs_string& out_ro_name,
                            const rs_string& pron_conv_fname,
                            const rs_string& pron_dict_fname,
                            bool keep,
                            const rs_string& affix_cg_fname,
                            const rs_string& proper_lc_fname,
                            const rs_string& sort_order,
                            bool common_only,
                            const rs_string& rflag,
                            const rs_string& kaffixes,
                            unsigned min_freqx,
                            const rs_string& debugw)
{
    cd_keep_to = keep;
    cd_common_only = common_only;
    cd_conv_dir = cdir;
    cd_ro_base_name = out_ro_name;
    cd_affix_cg_fname = affix_cg_fname;
    cd_proper_lc_fname = proper_lc_fname;
    cd_sort_order = sort_order;
    cd_rflag = rflag;
    cd_min_freqx = min_freqx;
    cd_debugw = debugw;
    string_to_vec (kaffixes, cd_kaffix_v, L",");
    cd_aff.rs_affix_init (affix_cg_fname, L"TO");
    if (!init_tables (bpd_name, pron_conv_fname, pron_dict_fname))
        return false;
    if (cd_ro_base_name.length() == 0)
        cd_ro_base_name = bpd_name;
    rs_string_v combining_v, used_v;
    if (options == L"+") {
        // all combinations of options to produce 
        // where exactly one transformation option is activated at a time
        rs_string_v all_v = cd_pob.get_all_options();
        unsigned opt_size = all_v.size(), i;
        for (i=0; i< opt_size; i++) {
            used_v.push_back (all_v[i]);
            create_rec (combining_v, used_v, false);
            used_v.clear();
        }
    } else if (options != L"") {
        // otherwise, options are interpreted as 'used*combining'
        // if no star character (*), combining=""
        // all possible combining combinations (at least one activated at a time)
        rs_string used_options =  nth_substring (options, 0, L"*");
        rs_string combining_options =  nth_substring (options, 1, L"*");
        string_to_vec (used_options, used_v, L",");
        string_to_vec (combining_options, combining_v, L",");
        create_rec (combining_v, used_v, combining_v.size()==0);
    }
    return true;
}


rs_string get_option_infix (const rs_string_v& opt_v)
{   // from the vector of options, removes all dashes "-" and merges all the strings
    // to get file name infix
    rs_string result, part;
    unsigned opt_size=opt_v.size(), i;
    for (i=0; i<opt_size; i++) {
        // remove dash "-" from 
        rs_string_v opt_iv;
        string_to_vec (opt_v[i], opt_iv, L"-");
        vec_to_string (opt_iv, part, L"");
        result += part;
    }
    return result;
}

bool is_group_present (const rs_string& group, const rs_string_v& opt_v)
{
    // returns true if "group" is already listed among any option_v
    unsigned opt_size=opt_v.size(), i;
    for (i=0; i<opt_size; i++) {
        rs_string opt_group = nth_substring (opt_v[i], 0, L"-");
        if (group == opt_group)
            return true;
    }
    return false;
}


void dcreate::create_rec (rs_string_v combining_v, // passing vector by value, by intention
                             rs_string_v used_v, // passing vector by value, by intention
                             bool plain_ro_dict_name)
{
    // recursively creates all the dictionaries 
    if (combining_v.size() == 0) {
        if (used_v.size() == 0)
            return; // do not repeat TO dictionary
        rs_string out_name_infix;
        if (!plain_ro_dict_name)
            out_name_infix = L"-" + get_option_infix (used_v); // todo skip options selected before "*" from used_v
        rs_log_file (L"out_name_infix: " + out_name_infix);
        cd_pob.enable_options (used_v);
        create_base (out_name_infix);
        return;
    }
    rs_string opt=combining_v[0];
    combining_v.erase (combining_v.begin());
    create_rec (combining_v, used_v, plain_ro_dict_name);
    rs_string opt_group = nth_substring (opt, 0, L"-");
    if (!is_group_present (opt_group, used_v)) {
        used_v.push_back (opt);
        create_rec (combining_v, used_v, plain_ro_dict_name);
    }
}


void dcreate::create_base (const rs_string& out_name_infix)
{
    // creates one dictionary from pronunciation dictionary, using specified options
    rs_string ro_name = cd_ro_base_name + out_name_infix;
    // set table file name
    rs_string out_fname = ro_name + L"_dict.rst";
    rs_log_console (L"Creating RO dictionary "+out_fname);
    //return; // for testing only
    rs_table out_rst (ro_name, L"respelled orthography "+ro_name, L"en-x-respelled-" + ro_name);
    // column indexes in output table - RO dictionary

    // copy columns from cd_pron_rst to out_rst
    out_rst.get_columns_from (cd_pron_rst);
    text_cix cix_or_to_us = out_rst.get_text_cix (L"OR_TO_US");
    text_cix cix_or_to_uk = out_rst.get_text_cix (L"OR_TO_UK");
    text_cix cix_or_pr = out_rst.get_text_cix (L"OR_PR");
    numb_cix cix_freq  = out_rst.get_numb_cix(L"FREQN");
    numb_cix cix_freql  = out_rst.get_numb_cix(L"FREQL");
    text_cix cix_flags = out_rst.get_text_cix(L"FLAGS");
    text_cix cix_stress = cd_pron_rst.get_text_cix (L"STRESS");
    text_cix cix_comp = cd_pron_rst.get_text_cix (L"COMP");
    
    text_cix cix_or_sp = out_rst.add_text_col (L"OR_SP", true, 16);
    numb_cix cix_nib_chan = out_rst.add_numb_col (L"NIB_CHAN", 2, false);
    numb_cix cix_nib_all = out_rst.add_numb_col (L"NIB_ALL", 2, false);
    //numb_cix cix_freqx = cd_pron_rst.get_numb_cix (L"FREQX");

    rs_rix row_count = cd_pron_rst.get_row_count ();
    unsigned ct_zero_freq=0, ct_prop_flag=0;
    unsigned max_word_len=0, this_word_len=0;
    for (rs_rix rix=rs_rix::begin(); rix<row_count; rix++) {
        const rs_string& word_to=cd_pron_rst.get_text (cix_or_to_us, rix);
        this_word_len = word_to.length();
        if (max_word_len < this_word_len)
            max_word_len = this_word_len;
    }

    unsigned word_len_ix;
    for (word_len_ix = 1;  word_len_ix < max_word_len;  word_len_ix++)
    {
        for (rs_rix rix=rs_rix::begin(); rix<row_count; rix++) {
            const rs_string& word_to=cd_pron_rst.get_text (cix_or_to_us, rix);
            //rs_log_file (L"processing word: " + word_to + L", current size: " + rs_stringed (word_len_ix));
            if (word_to.length() != word_len_ix) {
                //rs_log_file (L"skipping word due to size");
                continue;
            }
            rs_log_file (L"processing word: " + word_to + L", word size correct");
            //unsigned freqx_val = cd_pron_rst.get_numb (cix_freqx, rix);
            //if (freqx_val < cd_min_freqx) {
            //    ct_zero_freq++;
            //    rs_log_file (L"Skipping low-freqw word");
            //    continue;
            //}
            if (word_to == L"bath") {
                // this code is only to assist debugging
                rix = rix; // suitable place for breakpoint
            }

            rs_string word_ro, word_ro_specific;
            const rs_table& word_rst = cd_pob.get_word_rst();
            rs_rix word_rix = word_rst.get_text_row_any (word_to, cix_or_to_us);
            bool specific_word (false);
            if (word_rix.is_valid()) {
                word_ro_specific = word_rst.get_text (cix_or_pr, word_rix);
                if (word_ro_specific.empty())
                    word_ro_specific = word_to;
                specific_word = true;
            }
            unsigned freq_all = cd_pron_rst.get_numb (cix_freq, rix);
            if (cd_common_only && freq_all==0  && !specific_word) {
                ct_zero_freq++;
                rs_log_file (L"Skipping zero-frequency word:");
                continue;
            }
            rs_string old_flags=cd_pron_rst.get_text (cix_flags, rix);
            rs_string_v flag_v;
            string_to_vec (old_flags, flag_v, L",");
            if (!specific_word  &&
                find(flag_v.begin(), flag_v.end(), L"np") != flag_v.end()    &&
                find(flag_v.begin(), flag_v.end(), L"nc") == flag_v.end()) {
                    rs_log_file (L"Skipping np word");
                    ct_prop_flag++;
                    continue;
            }
            if (!cd_rflag.empty() && find(flag_v.begin(), flag_v.end(), cd_rflag) == flag_v.end()) {
                rs_log_file (L"Skipping word/flags without flag: " + word_to
                    + L"/" + old_flags
                    + L": missing " + cd_rflag);
                ct_prop_flag++;
                continue;
            }
            rs_log_file (L"Procesing word, passed all checks");
            rs_string word_rel=cd_pron_rst.get_text (cix_or_pr, rix);
            const rs_string& stress=cd_pron_rst.get_text (cix_stress, rix);
            const rs_string& word_comp=cd_pron_rst.get_text (cix_comp, rix); // composition
            rs_string word_base = nth_substring (word_comp, 0, L",");
            rs_string word_affix = nth_substring (word_comp, 1, L",");
            bool decomposable = false;
            unsigned nib_chan=0, nib_all=0; // nibble counts: changed and all
            if (word_affix != L""  &&  cd_kaffix_v.size()>0) {
                if (cd_kaffix_v[0] == L"*"  ||
                    find (cd_kaffix_v.begin(), cd_kaffix_v.end(), word_affix) != cd_kaffix_v.end())
                    decomposable = true;
            }
            if (word_to == cd_debugw)
            {
                nib_chan = nib_chan; // place for breakpoint
            }
            if (specific_word) {
                word_ro = word_ro_specific;
                old_flags = concat (L"fsw", old_flags, L',');
            } else if (decomposable) {
                rs_rix rix_base = out_rst.get_text_row_any (word_base, cix_or_to_us);
                if (rix_base.is_valid()) {
                    const rs_string& chan_base = out_rst.get_text (cix_or_sp, rix_base);
                    vector<size_t> af_vec;
                    size_t aff_ix = cd_aff.get_affix_ix (word_affix);
                    af_vec.push_back (aff_ix);
                    cd_aff.apply_affixes (word_ro, chan_base, af_vec);
                }
            }
            if (word_ro.empty()) {
                cd_pob.convert_to_ro (word_to, word_rel, word_ro, nib_chan, nib_all, stress, cd_pron_rst);
            }
            rs_string flag_value;
            if (word_to == word_ro)
                flag_value = L"fsm";
            else
                flag_value = L"fch";
            out_rst.append_text (word_to, cix_or_to_us);
            out_rst.append_text (cd_pron_rst.get_text (cix_or_to_uk, rix), cix_or_to_uk);
            //out_rst.append_text (cix_or_pr);
            out_rst.append_text (word_ro, cix_or_sp);
            out_rst.append_text (concat (flag_value, old_flags, L','), cix_flags);
            out_rst.append_numb (cd_pron_rst.get_numb (cix_freql, rix), cix_freql);
            out_rst.append_numb (cd_pron_rst.get_numb (cix_freq, rix), cix_freq);
            out_rst.append_numb (nib_chan, cix_nib_chan);
            out_rst.append_numb (nib_all, cix_nib_all);
        }
    }
    // in table out_rst, get duplicate lines where values in both columns
    // cix_or_to_us and cix_or_sp are equal
    // adds a column "duplicates" and sets value to 1 for duplicated rows
    // in a set of duplicate rows, the first row in not marked, all the next are marked
    numb_cix dup_cix  = out_rst.add_numb_col (L"DUPLICATES", 1, true);
    rs_rix end_rix = out_rst.get_row_count();
    rs_rix rix;
    unsigned ct_dupl=0;
    for (rix=rs_rix::begin(); rix < end_rix; rix++) {
        out_rst.set_numb (0, dup_cix, rix);
    }
    for (rix=rs_rix::begin(); rix < end_rix; rix++) {
        vector<rs_rix> rix_v;
        const rs_string& to_str = out_rst.get_text (cix_or_to_us, rix);
        const rs_string& ro_str = out_rst.get_text (cix_or_sp, rix);
        rix_v = out_rst.get_text_row_all (to_str, cix_or_to_us);
        unsigned i;
        for (i=0;  i<rix_v.size();  i++) {
            rs_rix rix2 = rix_v[i];
            if (rix2 <= rix)
                continue; // only searching for duplicates in the rest of tables
            if (out_rst.get_text (cix_or_sp, rix2)  ==  ro_str) {
                // found a duplicate and mark it
                out_rst.set_numb (1, dup_cix, rix2);
                rs_log_file (L"Marked duplicate (row2): to="+to_str+
                    L", ro="+ro_str+
                    L", row1="+rs_stringed(rix)+
                    L", row2="+rs_stringed(rix2));
                ct_dupl++;
            };
        }
    }
    // now get some statistics
    rs_log_file (L"List of changed words in orthography: ", ro_name);
    for (rix=rs_rix::begin(); rix < end_rix; rix++) {
        if (out_rst.get_numb (dup_cix, rix) == 1)
            continue; // duplicate, already counted
        const rs_string& flags=out_rst.get_text (cix_flags, rix);
        rs_string_v flag_v;
        string_to_vec (flags, flag_v, L",");
        unsigned flag_v_size = flag_v.size();
        bool fl_changed(false);
        for (unsigned i=0; i<flag_v_size; i++) {
            const rs_string& flag=flag_v[i];
            if (flag == L"fsm") {
                break;
            } else if (flag == L"fch") {
                fl_changed=true;
                break;
            }
        }
        if (fl_changed) {
            rs_log_file (L"changed word: "
                + out_rst.get_text (cix_or_to_us, rix) + L" "
                + out_rst.get_text (cix_or_sp, rix));
        }
    }
    rs_log_file (L"Changed words - end of list");
    rs_log_console (L"Totals of words: duplicates " + rs_stringed(ct_dupl)
        + L", low-freq skipped: " + rs_stringed (ct_zero_freq)
        + L", skipped np and rflag: " + rs_stringed (ct_prop_flag));

    out_rst.set_numbered (0);
    out_rst.write (out_fname,
        L"OR_TO_US,OR_TO_UK,OR_SP,FREQL,NIB_CHAN,NIB_ALL,FLAGS", L"DUPLICATES==0", cd_sort_order);

    get_ro_dict_stat (out_fname);
    create_homo_table (out_fname);
    if (cd_conv_dir.length() != 0) {
        // convert files from a specified directory
        converter& cnv = rs_command::get_appl().m_conversions.get_converter (
            L"TO_US", ro_name, cd_affix_cg_fname, cd_proper_lc_fname, false);
        cnv.convert_directory (cd_conv_dir, cd_conv_dir+L"-"+ro_name, cd_keep_to);
    }
}


bool dcreate::show_statistics (const rs_string& bpd_name,
                             const rs_string& pron_conv_fname,
                             const rs_string& pron_dict_fname,
                             const rs_string& sort,
                             const rs_string& coptions,
                             bool a_export)
{
    // gathers statistics on conversion options
    if (!init_tables (bpd_name, pron_conv_fname, pron_dict_fname))
        return false;

    const vector<ro_conv_rule>& conv_v = cd_pob.get_conv_v();
    const rs_string_v& option_v = cd_pob.get_all_options();
    unsigned j, opt_size=option_v.size();
    unsigned i, conv_size=conv_v.size();
    rs_rix row_count = cd_pron_rst.get_row_count ();
    unsigned total_freq = 0;  // number of all words
    rs_string_v coption_v;
    string_to_vec (coptions, coption_v, L",");

    // column indexes in cd_pron_rst
    text_cix cix_or_to_us = cd_pron_rst.get_text_cix (L"OR_TO_US");
    text_cix cix_or_sp = cd_pron_rst.get_text_cix (L"OR_PR");
    numb_cix cix_freq  = cd_pron_rst.get_numb_cix (L"FREQN");

    for (j=0; j<opt_size; j++) {
        // for all conversion options: j is index of conversion option
        //rs_log_console (L"option: " + option_v[j]);
        if (coption_v.size()>0 && 
                find (coption_v.begin(), coption_v.end(), option_v[j]) == coption_v.end()) {
            //rs_log_console (L"option not listed to be processed: " + option_v[j]);
            continue;
        }
        rs_string pron_stat_fname = bpd_name + L"_pron_stat-" + option_v[j] + 
            (a_export? L".txt" : L".rst");
        rs_table stat_rst;
        text_cix cix_ssearch = stat_rst.add_text_col (L"SEARCH", false, 15);
        text_cix cix_snibble = stat_rst.add_text_col (L"NIBBLE", false, 10);
        numb_cix cix_sfreq   = stat_rst.add_numb_col (L"FREQN", 10);
        text_cix cix_sor_to  = stat_rst.add_text_col (L"OR_TO", true, 15);
        text_cix cix_sor_rel = stat_rst.add_text_col (L"OR_REL", false, 15);
        numb_cix cix_sfirst   = stat_rst.add_numb_col (L"FIRST", 1); // flag: 1 if first row for given word
        rs_string_v active_v;
        active_v.push_back (option_v[j]);
        cd_pob.enable_options (active_v);
        rs_rix rix;
        bool rule_always_used = true;
        for (i=0; i<conv_size; i++) {
            // for all conversion rules: i is index of conversion rule
            if (conv_v[i].get_option() != option_v[j])
                continue;
            unsigned stat_count=0, stat_freq=0;
            for (rix=rs_rix::begin(); rix<row_count; rix++) {
                // for all words: count them and sum their frequencies
                rs_string word_to=cd_pron_rst.get_text (cix_or_to_us, rix);
                unsigned word_freq = cd_pron_rst.get_numb (cix_freq, rix);
                if (cd_common_only && word_freq==0) {
                    //rs_log_file (L"Skipping zero-frequency word: " + word_to);
                    continue;
                }
                rs_string word_rel=cd_pron_rst.get_text (cix_or_sp, rix);
                rs_string word_ro, used_nibble;
                bool conv_rule_used(false);
                cd_pob.convert_stat (word_to, word_rel, word_ro, i, conv_rule_used, used_nibble, cd_pron_rst);
                if (conv_rule_used) {
                    stat_rst.append_text (conv_v[i].search_str(), cix_ssearch);
                    stat_rst.append_text (used_nibble, cix_snibble);
                    stat_rst.append_numb (word_freq, cix_sfreq);
                    stat_rst.append_text (word_to, cix_sor_to);
                    stat_rst.append_text (word_rel, cix_sor_rel);
                    stat_rst.append_numb (0, cix_sfirst); // later are some changed to 1
                    stat_count++;
                    stat_freq+=word_freq;
                } else
                    rule_always_used = false;
            }
            wostringstream oss;
            oss << L"conversion: " << setw(15) << conv_v[i].search_str()
                << L": count=" << setw(7) << rs_stringed (stat_count, L".")
                << L", freq="  << setw(12) << rs_stringed (stat_freq, L".");
            //rs_log_console (oss.str());
        } // for i, conversion rules
        // now, let's filter results on first occurence of every word
        rs_rix stat_max_rix = stat_rst.get_row_count ();
        stat_rst.create_hashes ();
        for (rix=rs_rix::begin(); rix<stat_max_rix; rix++) {
            const rs_string& to_word = stat_rst.get_text (cix_sor_to, rix);
            vector<rs_rix> all_rix_v = stat_rst.get_text_row_all (to_word, cix_sor_to);
            rs_rix first_rix= all_rix_v[0];
            stat_rst.set_numb (1, cix_sfirst, first_rix);
        }
        unsigned stat_count=0, stat_freq=0;
        for (rix=rs_rix::begin(); rix<stat_max_rix; rix++) {
            bool is_first = stat_rst.get_numb (cix_sfirst, rix) != 0;
            if (is_first) {
                unsigned freq_uniq = stat_rst.get_numb (cix_sfreq, rix);
                stat_count++;
                stat_freq += freq_uniq;
            }
        }
        wostringstream oss;
        oss << L"Option: " << setw(10) << option_v[j]
            << L": count=" << setw(7) << rs_stringed (stat_count, L".")
            << L", freq="  << setw(12) << rs_stringed (stat_freq, L".");
        if (total_freq > 0) {
            double freq_ppm = (double) stat_freq / (double) total_freq * 1e6;
            oss << L" (" << setw(7) << rs_stringed ((unsigned) freq_ppm, L".") << L" ppm)";
        }
        oss << L", writing file: " << pron_stat_fname;
        rs_log_console (oss.str());
        if (rule_always_used) {
            total_freq = stat_freq;
        }
        stat_rst.write (pron_stat_fname, L"SEARCH,NIBBLE,FREQN,OR_TO,OR_REL", L"FIRST==1", sort,
            rs_rix(rs_rix::none()), !a_export);

    } // for j, conversion options
    return true;
}


void get_homo_words_and_freq (
    const rs_table& dict_rst,
    const vector<rs_rix>& rix_vec,
    const text_cix& cix,
    const numb_cix& freq_cix,
    rs_string& desc,
    unsigned& freq2
)
{
    vector<unsigned> freq2_vec;
    for (unsigned i=0; i<rix_vec.size(); i++) {
        unsigned freq = dict_rst.get_numb (freq_cix, rix_vec[i]);
        desc += (i>0?L",":L"") + 
            dict_rst.get_text (cix, rix_vec[i]) +
            L"(" + rs_stringed(freq) + L")";
        freq2_vec.push_back (freq);
    }
    sort (freq2_vec.begin(), freq2_vec.end());
    if (freq2_vec.size()>=2)
        freq2 = freq2_vec [freq2_vec.size() - 2];
    else
        freq2 = 0;
    // rs_log_file (L"get_homo_words_and_freq results: ",
    //    L"desc=" + desc + L", freq2="+rs_stringed(freq2));
}




void create_homo_table (const rs_string& dict_fname)
{
    // create table of homonyms and homographs
    // for the dictionary table with TO in column 0 and RO in column 1
    rs_table dict_rst (dict_fname);
    rs_table homo_rst;
    homo_rst.get_columns_from (dict_rst);
    numb_cix freq2_cix = homo_rst.add_numb_col (L"FREQ2", 8);
    text_cix desc_cix = homo_rst.add_text_col (L"DESC", false, 20);
    text_cix cix0 (0);
    text_cix cix1 (1);
    numb_cix freq_cix = dict_rst.get_numb_cix (L"FREQL");
    for (rs_rix rix=rs_rix::begin(); rix<dict_rst.get_row_count(); rix++) {
        const rs_string& word0 = dict_rst.get_text (cix0, rix);
        const rs_string& word1 = dict_rst.get_text (cix1, rix);
        vector<rs_rix> rix_vec0, rix_vec1;
        rix_vec0 = dict_rst.get_text_row_all (word0, cix0);
        rix_vec1 = dict_rst.get_text_row_all (word1, cix1);
        if (rix_vec0.size()>1  ||  rix_vec1.size()>1) {
            // word0 or word1 occur many times
            homo_rst.add_one_row (dict_rst, rix, dict_rst.get_col_count());
            rs_string desc0, desc1, desc;
            unsigned freq2_0=0, freq2_1=0, freq2;
            if (rix_vec0.size()>1) {
                get_homo_words_and_freq (dict_rst, rix_vec0, cix1, freq_cix,
                    desc0, freq2_0);
                desc0 = word0 + L"<->" + desc0;
            }
            if (rix_vec1.size()>1) {
                get_homo_words_and_freq (dict_rst, rix_vec1, cix0, freq_cix,
                    desc1, freq2_1);
                desc1 = desc1 + L"<->" + word1;
            }
            desc = desc0;
            if (desc.length()>0 && desc1.length() >0)
                desc+=L"|";
            desc+=desc1;
            freq2 = freq2_0 > freq2_1? freq2_0 : freq2_1;
            homo_rst.append_numb (freq2, freq2_cix);
            homo_rst.append_text (desc, desc_cix);
        }
    }
    homo_rst.set_col_min_width (12, cix0);
    homo_rst.set_col_min_width (12, cix1);
    rs_string out_fname (dict_fname);
    replace_file_substring (out_fname, L"_dict.rst", L"_homo.rst");
    rs_log_console (L"Creating table of homonyms and homographs: " + out_fname);
    homo_rst.write (out_fname, L"OR_TO_US,OR_TO_UK,OR_SP,FREQL,FREQ2,DESC", L"", L"+FREQ2");
}


bool get_ro_dict_stat (const rs_string& dict_fname)
{
    // outputs how many words/nibbles have been changed, counts, percentage, weighed percentage
    rs_table dict_rst (dict_fname);
    if (!dict_rst.is_valid())
        return false;
    rs_log_file (L"List of changed words in dictionary: ", dict_fname);
    rs_rix end_rix = dict_rst.get_row_count(), rix;
    text_cix cix0 (0);
    text_cix cix1 (1);
    numb_cix freq_cix = dict_rst.get_numb_cix (L"FREQL");
    numb_cix nib_chan_cix = dict_rst.get_numb_cix (L"NIB_CHAN");
    numb_cix nib_all_cix = dict_rst.get_numb_cix (L"NIB_ALL");
    text_cix cix_flags = dict_rst.get_text_cix (L"FLAGS");
    unsigned wd_ch=0, wd_tl=0, wd_ch_w=0, wd_tl_w=0; // words changed and total; also weigthed variants
    unsigned nb_ch=0, nb_tl=0, nb_ch_w=0, nb_tl_w=0; // nibbles changed and total; also weigthed variants
    unsigned freq; // weight for previous 
    unsigned nib_chan, nib_all;
    for (rix=rs_rix::begin(); rix < end_rix; rix++) {
        const rs_string& flags=dict_rst.get_text (cix_flags, rix);
        rs_string_v flag_v;
        string_to_vec (flags, flag_v, L",");
        unsigned flag_v_size = flag_v.size();
        bool fl_changed(false);
        for (unsigned i=0; i<flag_v_size; i++) {
            const rs_string& flag=flag_v[i];
            if (flag == L"fsm") {
                break;
            } else if (flag == L"fch") {
                fl_changed=true;
                break;
            }
        }
        freq = dict_rst.get_numb (freq_cix, rix);
        nib_chan = dict_rst.get_numb (nib_chan_cix, rix);
        nib_all = dict_rst.get_numb (nib_all_cix, rix);
        if (fl_changed) {
            wd_ch++;
            wd_ch_w += freq;
            nb_ch += nib_chan;
            nb_ch_w += nib_chan * freq;
        }
        wd_tl++;
        wd_tl_w += freq;
        nb_tl += nib_all;
        nb_tl_w += nib_all * freq;
    }
    double wd_perc = double (wd_ch) / double (wd_tl) * 100;
    double wd_perc_w = double (wd_ch_w) / double (wd_tl_w) * 100;
    rs_log_console (L"changed words:   "
        + rs_stringed (wd_perc) + L"% plain"
        + L" ("+rs_stringed (wd_ch) + L"/" + rs_stringed (wd_tl) +L")"
        + L", "+rs_stringed (wd_perc_w) + L"% weigthed"
        + L" ("+rs_stringed (wd_ch_w) + L"/" + rs_stringed (wd_tl_w) +L")"
        );
    double nb_perc = double (nb_ch) / double (nb_tl) * 100;
    double nb_perc_w = double (nb_ch_w) / double (nb_tl_w) * 100;
    rs_log_console (L"changed nibbles: "
        + rs_stringed (nb_perc) + L"% plain"
        + L" ("+rs_stringed (nb_ch) + L"/" + rs_stringed (nb_tl) +L")"
        + L", "+rs_stringed (nb_perc_w) + L"% weigthed"
        + L" ("+rs_stringed (nb_ch_w) + L"/" + rs_stringed (nb_tl_w) +L")"
        );
    return false;
}

void update_to_us_uk(const rs_string& in_dict_fname,
                     const rs_string& out_dict_fname,
                     const rs_string& column_us,
                     const rs_string& column_uk)
{
    // create table of homonyms and homographs
    // for the dictionary table with TO in column 0 and RO in column 1
    rs_table dict_rst (in_dict_fname);
    rs_table homo_rst;
    homo_rst.get_columns_from (dict_rst);
    numb_cix freq2_cix = homo_rst.add_numb_col (L"FREQ2", 8);
    text_cix desc_cix = homo_rst.add_text_col (L"DESC", false, 20);
    text_cix cix0 (0);
    text_cix cix1 (1);
    numb_cix freq_cix = dict_rst.get_numb_cix (L"FREQN");
    for (rs_rix rix=rs_rix::begin(); rix<dict_rst.get_row_count(); rix++) {
        const rs_string& word0 = dict_rst.get_text (cix0, rix);
        const rs_string& word1 = dict_rst.get_text (cix1, rix);
        vector<rs_rix> rix_vec0, rix_vec1;
        rix_vec0 = dict_rst.get_text_row_all (word0, cix0);
        rix_vec1 = dict_rst.get_text_row_all (word1, cix1);
        if (rix_vec0.size()>1  ||  rix_vec1.size()>1) {
            // word0 or word1 occur many times
            homo_rst.add_one_row (dict_rst, rix, dict_rst.get_col_count());
            rs_string desc0, desc1, desc;
            unsigned freq2_0=0, freq2_1=0, freq2;
            if (rix_vec0.size()>1) {
                get_homo_words_and_freq (dict_rst, rix_vec0, cix1, freq_cix,
                    desc0, freq2_0);
                desc0 = word0 + L"<->" + desc0;
            }
            if (rix_vec1.size()>1) {
                get_homo_words_and_freq (dict_rst, rix_vec1, cix0, freq_cix,
                    desc1, freq2_1);
                desc1 = desc1 + L"<->" + word1;
            }
            desc = desc0;
            if (desc.length()>0 && desc1.length() >0)
                desc+=L"|";
            desc+=desc1;
            freq2 = freq2_0 > freq2_1? freq2_0 : freq2_1;
            homo_rst.append_numb (freq2, freq2_cix);
            homo_rst.append_text (desc, desc_cix);
        }
    }
    homo_rst.set_col_min_width (12, cix0);
    homo_rst.set_col_min_width (12, cix1);
    rs_log_console (L"Writing updated table: " + out_dict_fname);
    homo_rst.write (out_dict_fname, L"", L"", L"-freq2");
}
